# -*- coding: utf-8 -*-


from fake_useragent import UserAgent
import requests
import base64
import json
from pymongo import MongoClient
import datetime

"""
国家税务总局-重大税收违法案件信息公布栏
http://hd.chinatax.gov.cn/nszx/InitMajor.html
"""

# 代理服务器
proxyHost = "http-dyn.abuyun.com"
proxyPort = "9020"
# 代理隧道验证信息
proxyUser = "H399CS954Q76448D"
proxyPass = "81084FEAADFAB080"


def get_comments():
    # http://hd.chinatax.gov.cn/nszx/InitMajor.html
    url = "http://hd.chinatax.gov.cn/service/findMajor.do"
    for num in range(52, 95):
        print("==============")
        print("开始：" + str(num))

        html = get_request(url, str(num), "2019年4月")
        data_json = json.loads(html.text)
        print("========开始解析数据=========")
        for d in data_json["content"]:
            detail_Dict = dict()
            detail_Dict["year"] = "2019年"
            detail_Dict["month"] = "4月"
            detail_Dict["name"] = d["name"]  # 企业名称
            detail_Dict["code"] = d.get("code")  # 纳税人识别号
            detail_Dict["caseNature"] = d["caseNature"]  # 案件性质
            detail_Dict["lawBasisTaxPunish"] = d["lawBasisTaxPunish"]    # 相关法律依据及
            detail_Dict["mainFacts"] = d["mainFacts"]    # 主要违法事实
            detail_Dict["mainFacts"] = d["mainFacts"] + d["lawBasisTaxPunish"]   # 主要违法事实 相关法律依据及
            detail_Dict["address"] = d.get("address")    # 地址
            detail_Dict["legalIdcardNumber"] = d.get("legalIdcardNumber")    # 身份证号码
            detail_Dict["legalIdcardType"] = d.get("legalIdcardType")    # 证件类型
            detail_Dict["legalName"] = d.get("legalName")    # 姓名
            detail_Dict["legalSex"] = d.get("legalSex")  # 性别
            detail_Dict["publish"] = d["publish"]    # 发布日期
            insertItem('chufa', detail_Dict)
        print("结束: " + str(num))
        print("===========")




def get_request(url, num, rq):
        try:
            ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
            cookies = {"yfx_c_g_u_id_10003701":"_ck19103017425611961702106597378",
                       "yfx_f_l_v_t_10003701":"f_t_1572428576175__r_t_1572428576175__v_t_1572428576175__r_c_0",
                       "_Jo0OQK":"B9181CA0A1702481B2751722B1BFAD95AE01A6E34095177BA87E391D6A2D3E4282EF36244ED1CD411D3251EB7E2DA4F00257D7E9348A0B2271CCAB7C96E1A2A2E9E1B918CCA8FE3BB9470EE0D297309F84070EE0D297309F840BD655D7AA310F3D66F0DFB7DA5A877C2GJ1Z1RQ==",
                       "JSESSIONID":"218E1D47D642389E4C497A0975827BBB"}
            headers = {
                "User-Agent": ua.random,
                "Accept": "*/*",
                "Accept-Encoding": "gzip, deflate",
                "Accept-Language": "zh-CN,zh;q=0.9",
                "Connection": "keep-alive",
                "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
                "DNT": "1",
                "Host": "hd.chinatax.gov.cn",
                "Origin": "http://hd.chinatax.gov.cn",
                "Referer": "http://hd.chinatax.gov.cn/nszx/InitMajor.html",
                "X-Requested-With": "XMLHttpRequest",
                "Proxy-Authorization": "Basic " + base64.b64encode((proxyUser + ":" + proxyPass).encode('utf-8'))
                    .decode()
            }
            postData = {
                "caseNature": "",
                "publish": rq,
                "page": num,
                "cPage": "",
                "categeryid": "1",
                "querystring1": "name",
                "querystring0": "name",
                "queryvalue": "",
            }
            proxyMeta = "http://%(user)s:%(pass)s@%(host)s:%(port)s" % {
                "host": proxyHost,
                "port": proxyPort,
                "user": proxyUser,
                "pass": proxyPass,
            }
            proxies = {
                "http": proxyMeta,
                "https": proxyMeta,
            }
            html = requests.post(url, data = postData, headers = headers, cookies = cookies)
            # html = requests.post(url, data = postData, proxies=proxies, headers = headers, cookies = cookies)
            if html.status_code != 200:
                print("=========重复请求:" + str(html.status_code) + "=========" + str(datetime.datetime.now()))
                get_request(url, num, rq)
            data_json = json.loads(html.text)
            if data_json.get("size") is None:
                get_request(url, num, rq)
        except Exception as ex:
            print("-------------访问错误------------")
            print(ex)
            return get_request(url, num, rq)
        return html

# 将cookies字符串转换dict格式
def get_cookie_dict(str):
    itemDict = {}
    items = str.split(';')
    for item in items:
        arr = item.split('=')
        key = arr[0].replace(' ', '')
        value = arr[1]
        itemDict[key] = value
    return itemDict


def insertItem(tableName, data):
    my_set = db[tableName]
    my_set.insert_one(data)


if __name__ == '__main__':
    conn = MongoClient('127.0.0.1', 27017)
    db = conn["swj"]
    get_comments()